# -*- coding: utf-8 -*-
"""
Created on 2021-07-30 09:31:05
---------
@summary:
---------
@author: xubin
"""

import time
import feapder
from feapder import Item

from crawler.utils import UpStatus, compress_content
from crawler.utils.get_task_form_mq import GetTask


class AddagenSpider(feapder.AirSpider):

    us = UpStatus()

    def start_requests(self):
        ct = GetTask()
        while True:
            if not ct.get_task_info():
                time.sleep(10)
                continue
            task_dic = eval(ct.get_task_info()['task_info'])
            url = task_dic['listpageurl']
            yield feapder.Request(url, task_info=task_dic)

    def parse(self, request, response):
        response.code = "utf-8"
        item = Item()
        html = response.extract()
        item.table_name = "Addgene"  # 指定存储的表名
        task_info = request.task_info

        item._id = task_info['id']
        item.length = len(html)
        item.filter_word = task_info['filterWord']
        item.url = task_info['listpageurl']
        item.createBy = 'xubin'
        item.updateTime = int(time.time())
        item.is_compress = True
        item.html = compress_content(html)

        yield item
        yield self.us.update_status(task_info['id'])


if __name__ == "__main__":
    AddagenSpider(thread_count=5).start()
